package edu.hit.crawler.http;

// JDK imports
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;

// Nutch imports
import org.apache.log4j.Logger;

import edu.hit.crawler.io.Response;
import edu.hit.crawler.util.MemcachedUtil;
import edu.hit.memcached.tool.MemCachedClient;

public class RawDownloader {

	private static Logger logger = Logger.getLogger(RawDownloader.class);

	/*
	 * static { PropertyConfigurator.configure("myLog4j.property");
	 * 
	 * }
	 */
	public final int BUFFER_SIZE = 8 * 1024;
	/** The network timeout in millisecond */
	protected int timeout = 10000;
	/** The length limit for downloaded content, in bytes. */
	protected int maxContent = 1024 * 1024;

	private byte[] content;
	private int code;
	private HashMap<String, String> headers = new HashMap<String, String>();
	private static MemCachedClient mcc = MemcachedUtil.getMemcachedClient();
	// 从缓存中获取ip地址，若不存在则getByname
	public InetAddress getAddress(String host) throws UnknownHostException {

		boolean flag = false;
		String result = null;

		try {
			if (null != mcc) {

				result = (String) mcc.get(host);
				if (null != result) {
					flag = true;
				//	System.out.println("hits: " + host + " " + result);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}

		if (!flag) {
			InetAddress addr = InetAddress.getByName(host);
			if (null != mcc)
				mcc.add(host, addr.getHostAddress());
			return addr;
		} else {

			String[] split = result.split("\\.");
			byte[] temp = new byte[4];
			for (int i = 0; i < 4; i++) {
				temp[i] = (byte) Integer.parseInt(split[i]);
			}
			return InetAddress.getByAddress(host, temp);
		}
	}

	public Response Download(URL url) throws HttpException, IOException {

		if (!"http".equals(url.getProtocol()))
			throw new HttpException("Not an HTTP url:" + url);


		String path = "".equals(url.getFile()) ? "/" : url.getFile();

		// some servers will redirect a request with a host line like
		// "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
		// don't want the :80...

		String host = url.getHost();
		int port;
		String portString;
		if (url.getPort() == -1) {
			port = 80;
			portString = "";
		} else {
			port = url.getPort();
			portString = ":" + port;
		}
		Socket socket = null;

		try {
			socket = new Socket(); // create the socket
			socket.setSoTimeout(timeout);

			// connect
			int sockPort = port;

			// get host address
			InetAddress addr = getAddress(host);
			InetSocketAddress sockAddr = null;
			if (addr != null)
				sockAddr = new InetSocketAddress(addr, sockPort);

			socket.connect(sockAddr, timeout);

			// make request
			OutputStream req = socket.getOutputStream();

			StringBuffer reqStr = new StringBuffer("GET ");

			reqStr.append(path);
			reqStr.append(" HTTP/1.0\r\n");
			reqStr.append("Host: ");
			reqStr.append(host);
			reqStr.append(portString);
			reqStr.append("\r\n");

			reqStr.append("Accept: text/html\r\n");
			reqStr.append("Accept-Encoding: x-gzip, gzip, deflate\r\n");

			String userAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; Maxthon 2.0)";
			reqStr.append("User-Agent: ");
			reqStr.append(userAgent);
			reqStr.append("\r\n");

			/*
			 * reqStr.append("Accept-Language: "); reqStr.append("zh, en");
			 * reqStr.append("\r\n");
			 */

			/*
			 * reqStr.append("If-Modified-Since: " + getModifiedTime());
			 * reqStr.append("\r\n");
			 */

			reqStr.append("\r\n");

			byte[] reqBytes = reqStr.toString().getBytes();

			req.write(reqBytes);
			req.flush();

			PushbackInputStream in = // process response
			new PushbackInputStream(new BufferedInputStream(
					socket.getInputStream(), BUFFER_SIZE), BUFFER_SIZE);

			StringBuffer line = new StringBuffer();

			boolean haveSeenNonContinueStatus = false;
			while (!haveSeenNonContinueStatus) {
				// parse status code line
				code = parseStatusLine(in, line);
				// System.out.println("statu code: " + code);
				// parse headers
				parseHeaders(in, line);
				haveSeenNonContinueStatus = code != 100; // 100 is "Continue"
			}

			// public final static String TRANSFER_ENCODING =
			// "Transfer-Encoding";
			/*
			 * String transferEncoding = getHeader("Transfer-Encoding");
			 * 
			 * if(transferEncoding != null &&
			 * transferEncoding.equalsIgnoreCase("chunked")){ line = new
			 * StringBuffer(); this.readChunkedContent(in, line); }else{
			 * readPlainContent(in); }
			 */

			readPlainContent(in);

			String contentEncoding = getHeader("Content-Encoding");
			if ("gzip".equals(contentEncoding)
					|| "x-gzip".equals(contentEncoding)) {
				content = Decode.processGzipEncoded(content, url);
			} else if ("deflate".equals(contentEncoding)) {
				content = Decode.processDeflateEncoded(content, url);
			} else {
				if (logger.isTraceEnabled()) {
					logger.trace("fetched " + content.length + " bytes from "
							+ url);
				}
			}

		} finally {
			if (socket != null)
				socket.close();
		}
		return new Response(url.toString(), code, content, headers);

	}

	/*
	 * ------------------------- * <implementation:Response> *
	 * -------------------------
	 */

	public int getCode() {
		return code;
	}

	public String getHeader(String name) {
		return (String) headers.get(name);
	}

	public byte[] getContent() {
		return content;
	}

	/*
	 * ------------------------- * <implementation:Response> *
	 * -------------------------
	 */

	private void readPlainContent(InputStream in) throws HttpException,
			IOException {

		int contentLength = Integer.MAX_VALUE; // get content length
		String contentLengthString = headers.get("Content-Length");
		if (contentLengthString != null) {
			contentLengthString = contentLengthString.trim();
			try {
				if (!contentLengthString.isEmpty())
					contentLength = Integer.parseInt(contentLengthString);
			} catch (NumberFormatException e) {
				throw new HttpException("bad content length: "
						+ contentLengthString);
			}
		}
		if (contentLength > maxContent) // limit
			contentLength = maxContent;

		ByteArrayOutputStream out = new ByteArrayOutputStream(BUFFER_SIZE);
		byte[] bytes = new byte[BUFFER_SIZE];
		int length = 0; // read content
		for (int i = in.read(bytes); i != -1 && length + i <= contentLength; i = in
				.read(bytes)) {

			out.write(bytes, 0, i);
			length += i;
		}
		content = out.toByteArray();
	}

	@SuppressWarnings("unused")
	private void readChunkedContent(PushbackInputStream in, StringBuffer line)
			throws HttpException, IOException {
		boolean doneChunks = false;
		int contentBytesRead = 0;
		byte[] bytes = new byte[BUFFER_SIZE];
		ByteArrayOutputStream out = new ByteArrayOutputStream(BUFFER_SIZE);

		while (!doneChunks) {
			if (logger.isTraceEnabled()) {
				logger.trace("Http: starting chunk");
			}

			readLine(in, line, false);

			String chunkLenStr;
			// if (LOG.isTraceEnabled()) { LOG.trace("chunk-header: '" + line +
			// "'"); }

			int pos = line.indexOf(";");
			if (pos < 0) {
				chunkLenStr = line.toString();
			} else {
				chunkLenStr = line.substring(0, pos);
				// if (LOG.isTraceEnabled()) { LOG.trace("got chunk-ext: " +
				// line.substring(pos+1)); }
			}
			chunkLenStr = chunkLenStr.trim();
			int chunkLen;
			try {
				chunkLen = Integer.parseInt(chunkLenStr, 16);
			} catch (NumberFormatException e) {
				throw new HttpException("bad chunk length: " + line.toString());
			}

			if (chunkLen == 0) {
				doneChunks = true;
				break;
			}

			if ((contentBytesRead + chunkLen) > maxContent)
				chunkLen = maxContent - contentBytesRead;

			// read one chunk
			int chunkBytesRead = 0;
			while (chunkBytesRead < chunkLen) {

				int toRead = (chunkLen - chunkBytesRead) < BUFFER_SIZE ? (chunkLen - chunkBytesRead)
						: BUFFER_SIZE;
				int len = in.read(bytes, 0, toRead);

				if (len == -1)
					throw new HttpException("chunk eof after "
							+ contentBytesRead + " bytes in successful chunks"
							+ " and " + chunkBytesRead + " in current chunk");

				// DANGER!!! Will printed GZIPed stuff right to your
				// terminal!
				// if (LOG.isTraceEnabled()) { LOG.trace("read: " + new
				// String(bytes, 0, len)); }

				out.write(bytes, 0, len);
				chunkBytesRead += len;
			}

			readLine(in, line, false);

		}

		if (!doneChunks) {
			if (contentBytesRead != maxContent)
				throw new HttpException(
						"chunk eof: !doneChunk && didn't max out");
			return;
		}

		content = out.toByteArray();
		parseHeaders(in, line);

	}

	private int parseStatusLine(PushbackInputStream in, StringBuffer line)
			throws IOException, HttpException {
		readLine(in, line, false);

		int codeStart = line.indexOf(" ");
		int codeEnd = line.indexOf(" ", codeStart + 1);

		// handle lines with no plaintext result code, ie:
		// "HTTP/1.1 200" vs "HTTP/1.1 200 OK"
		if (codeEnd == -1)
			codeEnd = line.length();

		int code;
		try {
			code = Integer.parseInt(line.substring(codeStart + 1, codeEnd));
		} catch (NumberFormatException e) {
			throw new HttpException("bad status line '" + line + "': "
					+ e.getMessage());
		}

		return code;
	}

	private void processHeaderLine(StringBuffer line) throws IOException,
			HttpException {

		int colonIndex = line.indexOf(":"); // key is up to colon
		if (colonIndex == -1) {
			int i;
			for (i = 0; i < line.length(); i++)
				if (!Character.isWhitespace(line.charAt(i)))
					break;
			if (i == line.length())
				return;
			throw new HttpException("No colon in header:" + line);
		}
		String key = line.substring(0, colonIndex);

		int valueStart = colonIndex + 1; // skip whitespace
		while (valueStart < line.length()) {
			int c = line.charAt(valueStart);
			if (c != ' ' && c != '\t')
				break;
			valueStart++;
		}
		String value = line.substring(valueStart);
		headers.put(key, value);
	}

	// Adds headers to our headers Metadata
	private void parseHeaders(PushbackInputStream in, StringBuffer line)
			throws IOException, HttpException {

		while (readLine(in, line, true) != 0) {

			// handle HTTP responses with missing blank line after headers
			int pos;
			if (((pos = line.indexOf("<!DOCTYPE")) != -1)
					|| ((pos = line.indexOf("<HTML")) != -1)
					|| ((pos = line.indexOf("<html")) != -1)) {

				in.unread(line.substring(pos).getBytes("UTF-8"));
				line.setLength(pos);

				try {
					// TODO: (CM) We don't know the header names here
					// since we're just handling them generically. It would
					// be nice to provide some sort of mapping function here
					// for the returned header names to the standard metadata
					// names in the ParseData class
					processHeaderLine(line);
				} catch (Exception e) {
					// fixme:
					e.printStackTrace();
				}
				return;
			}

			processHeaderLine(line);
		}
	}

	private int readLine(PushbackInputStream in, StringBuffer line,
			boolean allowContinuedLine) throws IOException {
		line.setLength(0);
		for (int c = in.read(); c != -1; c = in.read()) {
			switch (c) {
			case '\r':
				if (peek(in) == '\n') {
					in.read();
				}
			case '\n':
				if (line.length() > 0) {
					// at EOL -- check for continued line if the current
					// (possibly continued) line wasn't blank
					if (allowContinuedLine)
						switch (peek(in)) {
						case ' ':
						case '\t': // line is continued
							in.read();
							continue;
						}
				}
				return line.length(); // else complete
			default:
				line.append((char) c);
			}
		}
		throw new EOFException();
	}

	private int peek(PushbackInputStream in) throws IOException {
		int value = in.read();
		in.unread(value);
		return value;
	}

}